package spiderRobot;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import util.SpiderRobot;
import util.StatisticsWebTimes;

public class UrlSpider {

	public static void main(String[] args) {
		// 读取xml
		// 定义从xml文件中读出的参数
		String dBName = null;
		String dBHost = null;
		String searchUrlFirst = null;
		String searchUrlEnd= null;
		String codeNo= null;
		String head = null;
		int repeatTimes_URL = 0;
		int interval_URL= 0;
		int sourceNet = 0;
		String urlPattern = null;
		String search = null;
		// 开始读取xml文件
		SAXBuilder builder = new SAXBuilder(false);
		List sourceSiteParameterList = null;
		try {
			Document doc = builder.build("urlSpider.xml");
			Element parameter = doc.getRootElement();
			sourceSiteParameterList = parameter.getChildren("sourceSite");
		} catch (JDOMException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		// 一次循环拿到一个采集目标网站的配置也就是xml文件中的一个sourceSite
		int i = 0;
		for (Iterator iter = sourceSiteParameterList.iterator(); iter.hasNext();) {
			Element sourceSite = (Element) iter.next();
			dBName = sourceSite.getChildTextTrim("dBName");
			//System.out.println(dBName);
			dBHost = sourceSite.getAttributeValue("host");
			//System.out.println(dBHost);
			repeatTimes_URL = new Integer(sourceSite.getChildTextTrim("repeatTimesURL"));
			// System.out.println(repeatTimes_URL[i]);
			interval_URL = new Integer(sourceSite.getChildTextTrim("intervalURL"));
			// System.out.println(interval_URL[i]);
			urlPattern = sourceSite.getChildTextTrim("urlPattern");
			// System.out.println(urlPattern[i]);
			searchUrlFirst = sourceSite.getChildTextTrim("first");
			// System.out.println(searchUrlFirst[i]);
			searchUrlEnd = sourceSite.getChildTextTrim("end");
			// System.out.println(searchUrlEnd[i]);
			codeNo = sourceSite.getChildTextTrim("codeNo");
			sourceNet = new Integer(sourceSite.getChildTextTrim("sourceNet"));
			head = sourceSite.getChildTextTrim("head");
			i++;
		}
		// 抓取链接
		// SpiderRobotWithSpace和SpiderRobot功能类似，只是后者多了除去搜索页内容中空格和换行的功能
		SpiderRobot spiderRobot = new SpiderRobot();
		spiderRobot.setCodeNo(codeNo);
		spiderRobot.setdBName(dBName);
		spiderRobot.setEnd(searchUrlEnd);
		spiderRobot.setFirst(searchUrlFirst);
		spiderRobot.setHost(dBHost);
		spiderRobot.setInterval(interval_URL);
		spiderRobot.setRepeatTimes(repeatTimes_URL);
		spiderRobot.setUrlPattern(urlPattern);
		spiderRobot.setSourceNet(sourceNet);
		spiderRobot.setHead(head);
		spiderRobot.run();
		
		/*// 抓取完成后，统计每个源网站的出现
		System.out.println("现在开始运行网址统计");
		StatisticsWebTimes swt = new StatisticsWebTimes();
		swt.setdBName(dBName);
		swt.setHost(dBHost);
		swt.statisticsWebTimes();
		System.out.println("运行网址统计完成");*/
	}
}
